package engine

import (
	"crawler/fetcher"
	"log"
)

func Run(seeds ...Request)  {
	var requests []Request
    for _,i:=range seeds{
    	//log.Printf("got url:%s",i.UrL)
    	requests=append(requests,i)
	}
    for len(requests)>0 {
    	r:=requests[0]
    	requests=requests[1:]
        body,err:=fetcher.Fetch(r.UrL)
		if err !=nil {
			log.Printf("Fetcher error url:%s, eror:%v",r.UrL,err)
			continue
		}
        parseResult:=r.ParserFunc(body)
        requests=append(requests,parseResult.Requests...)

		for _,item:=range parseResult.Items{
			log.Printf("Got item:%v",item)
		}
	}

}